{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.757680Z",
     "start_time": "2019-01-11T16:14:04.111927Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import matchzoo as mz\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Pack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Structure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`matchzoo.DataPack` is a MatchZoo native data structure that most MatchZoo data handling processes build upon. A `matchzoo.DataPack` consists of three parts: `left`, `right` and `relation`, each one of is a `pandas.DataFrame`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.789018Z",
     "start_time": "2019-01-11T16:14:06.759774Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_pack = mz.datasets.toy.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.806493Z",
     "start_time": "2019-01-11T16:14:06.791056Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_left</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_left</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q1</th>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q5</th>\n",
       "      <td>how did apollo creed die</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q6</th>\n",
       "      <td>how long is the term for federal judges</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q7</th>\n",
       "      <td>how a beretta model 21 pistols magazines works</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_left\n",
       "id_left                                                   \n",
       "Q1                           how are glacier caves formed?\n",
       "Q2       How are the directions of the velocity and for...\n",
       "Q5                                how did apollo creed die\n",
       "Q6                 how long is the term for federal judges\n",
       "Q7          how a beretta model 21 pistols magazines works"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.left.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.821623Z",
     "start_time": "2019-01-11T16:14:06.809995Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_right</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D1-0</th>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-1</th>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-2</th>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-3</th>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D1-4</th>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_right\n",
       "id_right                                                   \n",
       "D1-0      A partly submerged glacier cave on Perito More...\n",
       "D1-1              The ice facade is approximately 60 m high\n",
       "D1-2              Ice formations in the Titlis glacier cave\n",
       "D1-3      A glacier cave is a cave formed within the ice...\n",
       "D1-4      Glacier caves are often called ice caves , but..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.right.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.836817Z",
     "start_time": "2019-01-11T16:14:06.825048Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left id_right  label\n",
       "0      Q1     D1-0    0.0\n",
       "1      Q1     D1-1    0.0\n",
       "2      Q1     D1-2    0.0\n",
       "3      Q1     D1-3    1.0\n",
       "4      Q1     D1-4    0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.relation.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The main reason for using a `matchzoo.DataPack` instead of `pandas.DataFrame` is efficiency: we save space from storing duplicate texts and save time from processing duplicate texts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-13T03:12:15.986951Z",
     "start_time": "2018-12-13T03:12:15.984894Z"
    }
   },
   "source": [
    "## FrameView"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, since a big table is easier to understand and manage, we provide the `frame` that merges three parts into a single `pandas.DataFrame` when called."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.863520Z",
     "start_time": "2019-01-11T16:14:06.838422Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    0.0  \n",
       "1          The ice facade is approximately 60 m high    0.0  \n",
       "2          Ice formations in the Titlis glacier cave    0.0  \n",
       "3  A glacier cave is a cave formed within the ice...    1.0  \n",
       "4  Glacier caves are often called ice caves , but...    0.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that `frame` is not a method, but a property that returns a `matchzoo.DataPack.FrameView` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.870469Z",
     "start_time": "2019-01-11T16:14:06.866625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matchzoo.data_pack.data_pack.DataPack.FrameView"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data_pack.frame)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This view reflects changes in the data pack, and can be called to create a `pandas.DataFrame` at any time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.876932Z",
     "start_time": "2019-01-11T16:14:06.872211Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frame = data_pack.frame\n",
    "data_pack.relation['label'] = data_pack.relation['label'] + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.896152Z",
     "start_time": "2019-01-11T16:14:06.878689Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    1.0  \n",
       "1          The ice facade is approximately 60 m high    1.0  \n",
       "2          Ice formations in the Titlis glacier cave    1.0  \n",
       "3  A glacier cave is a cave formed within the ice...    2.0  \n",
       "4  Glacier caves are often called ice caves , but...    1.0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slicing a DataPack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You may use `[]` to slice a `matchzoo.DataPack` similar to slicing a `list`. This also returns a shallow copy of the sliced data like slicing a `list`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.904770Z",
     "start_time": "2019-01-11T16:14:06.898326Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_slice = data_pack[5:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A sliced data pack's `relation` will directly reflect the slicing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.913260Z",
     "start_time": "2019-01-11T16:14:06.906078Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left id_right  label\n",
       "0      Q2     D2-0    1.0\n",
       "1      Q2     D2-1    1.0\n",
       "2      Q2     D2-2    1.0\n",
       "3      Q2     D2-3    1.0\n",
       "4      Q2     D2-4    1.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.relation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, `left` and `right` will be processed so only relevant information are kept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.920205Z",
     "start_time": "2019-01-11T16:14:06.914881Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_left</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_left</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_left\n",
       "id_left                                                   \n",
       "Q2       How are the directions of the velocity and for..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.926853Z",
     "start_time": "2019-01-11T16:14:06.921604Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_right</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>D2-0</th>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-1</th>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-2</th>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-3</th>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D2-4</th>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text_right\n",
       "id_right                                                   \n",
       "D2-0      In physics , circular motion is a movement of ...\n",
       "D2-1      It can be uniform, with constant angular rate ...\n",
       "D2-2      The rotation around a fixed axis of a three-di...\n",
       "D2-3      The equations of motion describe the movement ...\n",
       "D2-4      Examples of circular motion include: an artifi..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.right"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to slice a frame view object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.942513Z",
     "start_time": "2019-01-11T16:14:06.928035Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0      Q2  How are the directions of the velocity and for...     D2-0   \n",
       "1      Q2  How are the directions of the velocity and for...     D2-1   \n",
       "2      Q2  How are the directions of the velocity and for...     D2-2   \n",
       "3      Q2  How are the directions of the velocity and for...     D2-3   \n",
       "4      Q2  How are the directions of the velocity and for...     D2-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  In physics , circular motion is a movement of ...    1.0  \n",
       "1  It can be uniform, with constant angular rate ...    1.0  \n",
       "2  The rotation around a fixed axis of a three-di...    1.0  \n",
       "3  The equations of motion describe the movement ...    1.0  \n",
       "4  Examples of circular motion include: an artifi...    1.0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.frame[5:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And this is equivalent to slicing the data pack first, then the frame, since both of them are based on the `relation` column. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.966465Z",
     "start_time": "2019-01-11T16:14:06.943913Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_left  text_left  id_right  text_right  label\n",
       "0     True       True      True        True   True\n",
       "1     True       True      True        True   True\n",
       "2     True       True      True        True   True\n",
       "3     True       True      True        True   True\n",
       "4     True       True      True        True   True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.frame() == data_pack.frame[5:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Slicing is extremely useful for partitioning data for training vs testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:06.977489Z",
     "start_time": "2019-01-11T16:14:06.968230Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_train = int(len(data_pack) * 0.8)\n",
    "data_pack.shuffle(inplace=True)\n",
    "train_slice = data_pack[:num_train]\n",
    "test_slice = data_pack[num_train:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforming Texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `apply_on_text` to transform texts in a `matchzoo.DataPack`. Check the documentation for more information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.004154Z",
     "start_time": "2019-01-11T16:14:06.978951Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with len: 100%|██████████| 1/1 [00:00<00:00, 1257.66it/s]\n",
      "Processing text_right with len: 100%|██████████| 5/5 [00:00<00:00, 13148.29it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>99</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left  text_left id_right  text_right  label\n",
       "0      Q2         85     D2-0         126    1.0\n",
       "1      Q2         85     D2-1         128    1.0\n",
       "2      Q2         85     D2-2          99    1.0\n",
       "3      Q2         85     D2-3          78    1.0\n",
       "4      Q2         85     D2-4         312    1.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.apply_on_text(len).frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.032299Z",
     "start_time": "2019-01-11T16:14:07.009179Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing left_length with len: 100%|██████████| 1/1 [00:00<00:00, 2790.62it/s]\n",
      "Processing right_length with len: 100%|██████████| 5/5 [00:00<00:00, 6510.87it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>left_length</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>right_length</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "      <td>126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "      <td>99</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "      <td>78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left  left_length  \\\n",
       "0      Q2  How are the directions of the velocity and for...           85   \n",
       "1      Q2  How are the directions of the velocity and for...           85   \n",
       "2      Q2  How are the directions of the velocity and for...           85   \n",
       "3      Q2  How are the directions of the velocity and for...           85   \n",
       "4      Q2  How are the directions of the velocity and for...           85   \n",
       "\n",
       "  id_right                                         text_right  right_length  \\\n",
       "0     D2-0  In physics , circular motion is a movement of ...           126   \n",
       "1     D2-1  It can be uniform, with constant angular rate ...           128   \n",
       "2     D2-2  The rotation around a fixed axis of a three-di...            99   \n",
       "3     D2-3  The equations of motion describe the movement ...            78   \n",
       "4     D2-4  Examples of circular motion include: an artifi...           312   \n",
       "\n",
       "   label  \n",
       "0    1.0  \n",
       "1    1.0  \n",
       "2    1.0  \n",
       "3    1.0  \n",
       "4    1.0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.apply_on_text(len, rename=('left_length', 'right_length')).frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since adding a column indicating text length is a quite common usage, you may simply do:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.059930Z",
     "start_time": "2019-01-11T16:14:07.037004Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing length_left with len: 100%|██████████| 1/1 [00:00<00:00, 1287.39it/s]\n",
      "Processing length_right with len: 100%|██████████| 5/5 [00:00<00:00, 5793.24it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>length_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>length_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-0</td>\n",
       "      <td>In physics , circular motion is a movement of ...</td>\n",
       "      <td>126</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-1</td>\n",
       "      <td>It can be uniform, with constant angular rate ...</td>\n",
       "      <td>128</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-2</td>\n",
       "      <td>The rotation around a fixed axis of a three-di...</td>\n",
       "      <td>99</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-3</td>\n",
       "      <td>The equations of motion describe the movement ...</td>\n",
       "      <td>78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q2</td>\n",
       "      <td>How are the directions of the velocity and for...</td>\n",
       "      <td>85</td>\n",
       "      <td>D2-4</td>\n",
       "      <td>Examples of circular motion include: an artifi...</td>\n",
       "      <td>312</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left  length_left  \\\n",
       "0      Q2  How are the directions of the velocity and for...           85   \n",
       "1      Q2  How are the directions of the velocity and for...           85   \n",
       "2      Q2  How are the directions of the velocity and for...           85   \n",
       "3      Q2  How are the directions of the velocity and for...           85   \n",
       "4      Q2  How are the directions of the velocity and for...           85   \n",
       "\n",
       "  id_right                                         text_right  length_right  \\\n",
       "0     D2-0  In physics , circular motion is a movement of ...           126   \n",
       "1     D2-1  It can be uniform, with constant angular rate ...           128   \n",
       "2     D2-2  The rotation around a fixed axis of a three-di...            99   \n",
       "3     D2-3  The equations of motion describe the movement ...            78   \n",
       "4     D2-4  Examples of circular motion include: an artifi...           312   \n",
       "\n",
       "   label  \n",
       "0    1.0  \n",
       "1    1.0  \n",
       "2    1.0  \n",
       "3    1.0  \n",
       "4    1.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_slice.append_text_length().frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To one-hot encode the labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.084057Z",
     "start_time": "2019-01-11T16:14:07.061948Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q16</td>\n",
       "      <td>how much is 1 tablespoon of water</td>\n",
       "      <td>D16-2</td>\n",
       "      <td>In the US and parts of Canada, a tablespoon is...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q12</td>\n",
       "      <td>how big did girl scout cookie boxes used to be</td>\n",
       "      <td>D12-1</td>\n",
       "      <td>This mound contains 74 boxes of cookies</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q5</td>\n",
       "      <td>how did apollo creed die</td>\n",
       "      <td>D5-0</td>\n",
       "      <td>Apollo Creed is a fictional character from the...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q15</td>\n",
       "      <td>how are pointe shoes made</td>\n",
       "      <td>D15-4</td>\n",
       "      <td>They are normally worn by female dancers, thou...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q14</td>\n",
       "      <td>how big do sebaceous cysts get</td>\n",
       "      <td>D14-3</td>\n",
       "      <td>Furthermore, under the microscope neither enti...</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                       text_left id_right  \\\n",
       "0     Q16               how much is 1 tablespoon of water    D16-2   \n",
       "1     Q12  how big did girl scout cookie boxes used to be    D12-1   \n",
       "2      Q5                        how did apollo creed die     D5-0   \n",
       "3     Q15                       how are pointe shoes made    D15-4   \n",
       "4     Q14                  how big do sebaceous cysts get    D14-3   \n",
       "\n",
       "                                          text_right      label  \n",
       "0  In the US and parts of Canada, a tablespoon is...  [0, 1, 0]  \n",
       "1            This mound contains 74 boxes of cookies  [0, 1, 0]  \n",
       "2  Apollo Creed is a fictional character from the...  [0, 1, 0]  \n",
       "3  They are normally worn by female dancers, thou...  [0, 1, 0]  \n",
       "4  Furthermore, under the microscope neither enti...  [0, 1, 0]  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.relation['label'] = data_pack.relation['label'].astype(int)\n",
    "data_pack.one_hot_encode_label(num_classes=3).frame().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Your own DataPack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use `matchzoo.pack` to build your own data pack. Check documentation for more information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.106635Z",
     "start_time": "2019-01-11T16:14:07.086306Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-0</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L-1</td>\n",
       "      <td>R</td>\n",
       "      <td>R-1</td>\n",
       "      <td>r</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L-2</td>\n",
       "      <td>S</td>\n",
       "      <td>R-2</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-3</td>\n",
       "      <td>t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-4</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>L-1</td>\n",
       "      <td>R</td>\n",
       "      <td>R-5</td>\n",
       "      <td>n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>L-2</td>\n",
       "      <td>S</td>\n",
       "      <td>R-6</td>\n",
       "      <td>u</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>L-0</td>\n",
       "      <td>A</td>\n",
       "      <td>R-2</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left text_left id_right text_right\n",
       "0     L-0         A      R-0          a\n",
       "1     L-1         R      R-1          r\n",
       "2     L-2         S      R-2          s\n",
       "3     L-0         A      R-3          t\n",
       "4     L-0         A      R-4          e\n",
       "5     L-1         R      R-5          n\n",
       "6     L-2         S      R-6          u\n",
       "7     L-0         A      R-2          s"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame({\n",
    "    'text_left': list('ARSAARSA'),\n",
    "    'text_right': list('arstenus')\n",
    "})\n",
    "my_pack = mz.pack(data)\n",
    "my_pack.frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unpack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Format data in a way so that MatchZoo models can directly fit it. For more details, consult `matchzoo/tutorials/models.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.120611Z",
     "start_time": "2019-01-11T16:14:07.108309Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x, y = data_pack[:3].unpack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.125694Z",
     "start_time": "2019-01-11T16:14:07.122061Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id_left': array(['Q16', 'Q12', 'Q5'], dtype='<U3'),\n",
       " 'text_left': array(['how much is 1 tablespoon of water',\n",
       "        'how big did girl scout cookie boxes used to be',\n",
       "        'how did apollo creed die'], dtype='<U46'),\n",
       " 'id_right': array(['D16-2', 'D12-1', 'D5-0'], dtype='<U5'),\n",
       " 'text_right': array(['In the US and parts of Canada, a tablespoon is the largest type of spoon used for eating from a bowl.',\n",
       "        'This mound contains 74 boxes of cookies',\n",
       "        'Apollo Creed is a fictional character from the Rocky films , initially portrayed as the Undisputed Heavyweight Champion of the World.'],\n",
       "       dtype='<U133')}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.130289Z",
     "start_time": "2019-01-11T16:14:07.127057Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1],\n",
       "       [1],\n",
       "       [1]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MatchZoo incorporates various datasets that can be loaded as MatchZoo native data structures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.135431Z",
     "start_time": "2019-01-11T16:14:07.131623Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['toy', 'wiki_qa', 'embeddings', 'snli']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.datasets.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The toy dataset doesn't need to be downloaded and can be directly used. It's the best choice to get things rolling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.158058Z",
     "start_time": "2019-01-11T16:14:07.136917Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    0.0  \n",
       "1          The ice facade is approximately 60 m high    0.0  \n",
       "2          Ice formations in the Titlis glacier cave    0.0  \n",
       "3  A glacier cave is a cave formed within the ice...    1.0  \n",
       "4  Glacier caves are often called ice caves , but...    0.0  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toy_train_rank = mz.datasets.toy.load_data()\n",
    "toy_train_rank.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.183102Z",
     "start_time": "2019-01-11T16:14:07.159705Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>[0, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>[1, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right   label  \n",
       "0  A partly submerged glacier cave on Perito More...  [1, 0]  \n",
       "1          The ice facade is approximately 60 m high  [1, 0]  \n",
       "2          Ice formations in the Titlis glacier cave  [1, 0]  \n",
       "3  A glacier cave is a cave formed within the ice...  [0, 1]  \n",
       "4  Glacier caves are often called ice caves , but...  [1, 0]  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toy_dev_classification, classes = mz.datasets.toy.load_data(stage='train', task='classification')\n",
    "toy_dev_classification.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.188920Z",
     "start_time": "2019-01-11T16:14:07.185003Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[False, True]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other larger datasets will be automatically downloaded the first time you use it. Run the following lines to trigger downloading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:07.393884Z",
     "start_time": "2019-01-11T16:14:07.190528Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-0</td>\n",
       "      <td>Cross section of sclerenchyma fibers in plant ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-1</td>\n",
       "      <td>Microscopic view of a histologic specimen of h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-2</td>\n",
       "      <td>In Biology , Tissue is a cellular organization...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-3</td>\n",
       "      <td>A tissue is an ensemble of similar cells from ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q8</td>\n",
       "      <td>How are epithelial tissues joined together?</td>\n",
       "      <td>D8-4</td>\n",
       "      <td>Organs are then formed by the functional group...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                    text_left id_right  \\\n",
       "0      Q8  How are epithelial tissues joined together?     D8-0   \n",
       "1      Q8  How are epithelial tissues joined together?     D8-1   \n",
       "2      Q8  How are epithelial tissues joined together?     D8-2   \n",
       "3      Q8  How are epithelial tissues joined together?     D8-3   \n",
       "4      Q8  How are epithelial tissues joined together?     D8-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  Cross section of sclerenchyma fibers in plant ...      0  \n",
       "1  Microscopic view of a histologic specimen of h...      0  \n",
       "2  In Biology , Tissue is a cellular organization...      0  \n",
       "3  A tissue is an ensemble of similar cells from ...      0  \n",
       "4  Organs are then formed by the functional group...      0  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wiki_dev_entailment_rank = mz.datasets.wiki_qa.load_data(stage='dev')\n",
    "wiki_dev_entailment_rank.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:12.746972Z",
     "start_time": "2019-01-11T16:14:07.395380Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L-0</td>\n",
       "      <td>This church choir sings to the masses as they ...</td>\n",
       "      <td>R-0</td>\n",
       "      <td>The church has cracks in the ceiling.</td>\n",
       "      <td>[0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L-0</td>\n",
       "      <td>This church choir sings to the masses as they ...</td>\n",
       "      <td>R-1</td>\n",
       "      <td>The church is filled with song.</td>\n",
       "      <td>[1, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L-0</td>\n",
       "      <td>This church choir sings to the masses as they ...</td>\n",
       "      <td>R-2</td>\n",
       "      <td>A choir singing at a baseball game.</td>\n",
       "      <td>[0, 1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L-1</td>\n",
       "      <td>A woman with a green headscarf, blue shirt and...</td>\n",
       "      <td>R-3</td>\n",
       "      <td>The woman is young.</td>\n",
       "      <td>[0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L-1</td>\n",
       "      <td>A woman with a green headscarf, blue shirt and...</td>\n",
       "      <td>R-4</td>\n",
       "      <td>The woman is very happy.</td>\n",
       "      <td>[1, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0     L-0  This church choir sings to the masses as they ...      R-0   \n",
       "1     L-0  This church choir sings to the masses as they ...      R-1   \n",
       "2     L-0  This church choir sings to the masses as they ...      R-2   \n",
       "3     L-1  A woman with a green headscarf, blue shirt and...      R-3   \n",
       "4     L-1  A woman with a green headscarf, blue shirt and...      R-4   \n",
       "\n",
       "                              text_right         label  \n",
       "0  The church has cracks in the ceiling.  [0, 0, 1, 0]  \n",
       "1        The church is filled with song.  [1, 0, 0, 0]  \n",
       "2    A choir singing at a baseball game.  [0, 1, 0, 0]  \n",
       "3                    The woman is young.  [0, 0, 1, 0]  \n",
       "4               The woman is very happy.  [1, 0, 0, 0]  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snli_test_classification, classes = mz.datasets.snli.load_data(stage='test', task='classification')\n",
    "snli_test_classification.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:12.752323Z",
     "start_time": "2019-01-11T16:14:12.748937Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['entailment', 'contradiction', 'neutral', '-']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T09:59:51.361693Z",
     "start_time": "2019-01-11T09:59:51.359438Z"
    }
   },
   "source": [
    "## Preprocessors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`matchzoo.preprocessors` are responsible for transforming data into correct forms that `matchzoo.models`. `BasicPreprocessor` is used for models with common forms, and some other models have customized preprocessors made just for them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:12.757952Z",
     "start_time": "2019-01-11T16:14:12.753707Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[matchzoo.preprocessors.dssm_preprocessor.DSSMPreprocessor,\n",
       " matchzoo.preprocessors.naive_preprocessor.NaivePreprocessor,\n",
       " matchzoo.preprocessors.basic_preprocessor.BasicPreprocessor,\n",
       " matchzoo.preprocessors.cdssm_preprocessor.CDSSMPreprocessor]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.preprocessors.list_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When in doubt, use the default preprocessor a model class provides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:12.762696Z",
     "start_time": "2019-01-11T16:14:12.759463Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.models.Naive.get_default_preprocessor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A preprocessor should be used in two steps. First, `fit`, then, `transform`. `fit` collects information into `context`, which includes everything the preprocessor needs to `transform` together with other useful information for later use. `fit` will only change the preprocessor's inner state but not the input data. In contrast, `transform` returns a modified copy of the input data without changing the preprocessor's inner state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:12.865844Z",
     "start_time": "2019-01-11T16:14:12.764773Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 13/13 [00:00<00:00, 709.00it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit: 100%|██████████| 100/100 [00:00<00:00, 3367.46it/s]\n",
      "Processing text_left with extend: 100%|██████████| 13/13 [00:00<00:00, 7770.55it/s]\n",
      "Processing text_right with extend: 100%|██████████| 100/100 [00:00<00:00, 103845.11it/s]\n",
      "Building VocabularyUnit from a datapack.: 100%|██████████| 2275/2275 [00:00<00:00, 982702.53it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'vocab_unit': <matchzoo.processor_units.processor_units.VocabularyUnit at 0x135ce1828>}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_raw = mz.datasets.toy.load_data('train', 'ranking')\n",
    "test_raw = mz.datasets.toy.load_data('test', 'ranking')\n",
    "preprocessor.fit(train_raw)\n",
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:12.929472Z",
     "start_time": "2019-01-11T16:14:12.868425Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 13/13 [00:00<00:00, 3735.68it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 100/100 [00:00<00:00, 2631.64it/s]\n",
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 3/3 [00:00<00:00, 2508.56it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit => PuncRemovalUnit => VocabularyUnit => FixedLengthUnit: 100%|██████████| 20/20 [00:00<00:00, 2855.50it/s]\n"
     ]
    }
   ],
   "source": [
    "train_preprocessed = preprocessor.transform(train_raw)\n",
    "test_preprocessed = preprocessor.transform(test_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.330491Z",
     "start_time": "2019-01-11T16:14:12.931614Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter \"name\" set to Naive.\n",
      "Parameter \"task\" set to Ranking Task.\n",
      "Parameter \"input_shapes\" set to [(30,), (30,)].\n",
      "Parameter \"optimizer\" set to adam.\n",
      "Epoch 1/1\n",
      "100/100 [==============================] - 0s 1ms/step - loss: 143040.0781\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mae': array([137.50372  , 581.2816   , 218.03787  , 327.0222   , 442.00424  ,\n",
       "        313.24966  , 320.42343  , 436.7562   ,  92.216965 ,  83.25445  ,\n",
       "        434.78665  , 678.2655   ,   5.0753255, 366.75586  , 535.05304  ,\n",
       "        222.2565   ,  44.238594 , 373.13083  , 330.38757  , 227.82645  ],\n",
       "       dtype=float32)}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = mz.models.Naive()\n",
    "model.guess_and_fill_missing_params()\n",
    "model.build()\n",
    "model.compile()\n",
    "x_train, y_train = train_preprocessed.unpack()\n",
    "model.fit(x_train, y_train)\n",
    "x_test, y_test = test_preprocessed.unpack()\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Processor Units"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessors utilize `mz.processor_units` to transform data. Processor units correspond to specific transformations and you may use them independently to preprocess a data pack. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.362242Z",
     "start_time": "2019-01-11T16:14:13.332398Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>A partly submerged glacier cave on Perito More...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>The ice facade is approximately 60 m high</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>Ice formations in the Titlis glacier cave</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>A glacier cave is a cave formed within the ice...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>how are glacier caves formed?</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>Glacier caves are often called ice caves , but...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                      text_left id_right  \\\n",
       "0      Q1  how are glacier caves formed?     D1-0   \n",
       "1      Q1  how are glacier caves formed?     D1-1   \n",
       "2      Q1  how are glacier caves formed?     D1-2   \n",
       "3      Q1  how are glacier caves formed?     D1-3   \n",
       "4      Q1  how are glacier caves formed?     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  A partly submerged glacier cave on Perito More...    0.0  \n",
       "1          The ice facade is approximately 60 m high    0.0  \n",
       "2          Ice formations in the Titlis glacier cave    0.0  \n",
       "3  A glacier cave is a cave formed within the ice...    1.0  \n",
       "4  Glacier caves are often called ice caves , but...    0.0  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack = mz.datasets.toy.load_data()\n",
    "data_pack.frame().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.409761Z",
     "start_time": "2019-01-11T16:14:13.363868Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 4283.26it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 5000.00it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[A, partly, submerged, glacier, cave, on, Peri...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[The, ice, facade, is, approximately, 60, m, h...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[Ice, formations, in, the, Titlis, glacier, cave]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[A, glacier, cave, is, a, cave, formed, within...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[Glacier, caves, are, often, called, ice, cave...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                              text_left id_right  \\\n",
       "0      Q1  [how, are, glacier, caves, formed, ?]     D1-0   \n",
       "1      Q1  [how, are, glacier, caves, formed, ?]     D1-1   \n",
       "2      Q1  [how, are, glacier, caves, formed, ?]     D1-2   \n",
       "3      Q1  [how, are, glacier, caves, formed, ?]     D1-3   \n",
       "4      Q1  [how, are, glacier, caves, formed, ?]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [A, partly, submerged, glacier, cave, on, Peri...    0.0  \n",
       "1  [The, ice, facade, is, approximately, 60, m, h...    0.0  \n",
       "2  [Ice, formations, in, the, Titlis, glacier, cave]    0.0  \n",
       "3  [A, glacier, cave, is, a, cave, formed, within...    1.0  \n",
       "4  [Glacier, caves, are, often, called, ice, cave...    0.0  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = mz.processor_units.TokenizeUnit()\n",
    "data_pack.apply_on_text(tokenizer.transform, inplace=True)\n",
    "data_pack.frame[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.436648Z",
     "start_time": "2019-01-11T16:14:13.412498Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 17807.30it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 44758.34it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[a, partly, submerged, glacier, cave, on, peri...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[the, ice, facade, is, approximately, 60, m, h...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[ice, formations, in, the, titlis, glacier, cave]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[a, glacier, cave, is, a, cave, formed, within...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[glacier, caves, are, often, called, ice, cave...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                              text_left id_right  \\\n",
       "0      Q1  [how, are, glacier, caves, formed, ?]     D1-0   \n",
       "1      Q1  [how, are, glacier, caves, formed, ?]     D1-1   \n",
       "2      Q1  [how, are, glacier, caves, formed, ?]     D1-2   \n",
       "3      Q1  [how, are, glacier, caves, formed, ?]     D1-3   \n",
       "4      Q1  [how, are, glacier, caves, formed, ?]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [a, partly, submerged, glacier, cave, on, peri...    0.0  \n",
       "1  [the, ice, facade, is, approximately, 60, m, h...    0.0  \n",
       "2  [ice, formations, in, the, titlis, glacier, cave]    0.0  \n",
       "3  [a, glacier, cave, is, a, cave, formed, within...    1.0  \n",
       "4  [glacier, caves, are, often, called, ice, cave...    0.0  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lower_caser = mz.processor_units.LowercaseUnit()\n",
    "data_pack.apply_on_text(lower_caser.transform, inplace=True)\n",
    "data_pack.frame[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or use `chain_transform` to apply multiple processor units at one time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.488934Z",
     "start_time": "2019-01-11T16:14:13.438244Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of TokenizeUnit => LowercaseUnit: 100%|██████████| 13/13 [00:00<00:00, 5539.00it/s]\n",
      "Processing text_right with chain_transform of TokenizeUnit => LowercaseUnit: 100%|██████████| 100/100 [00:00<00:00, 5056.36it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[a, partly, submerged, glacier, cave, on, peri...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[the, ice, facade, is, approximately, 60, m, h...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[ice, formations, in, the, titlis, glacier, cave]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[a, glacier, cave, is, a, cave, formed, within...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[how, are, glacier, caves, formed, ?]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[glacier, caves, are, often, called, ice, cave...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                              text_left id_right  \\\n",
       "0      Q1  [how, are, glacier, caves, formed, ?]     D1-0   \n",
       "1      Q1  [how, are, glacier, caves, formed, ?]     D1-1   \n",
       "2      Q1  [how, are, glacier, caves, formed, ?]     D1-2   \n",
       "3      Q1  [how, are, glacier, caves, formed, ?]     D1-3   \n",
       "4      Q1  [how, are, glacier, caves, formed, ?]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [a, partly, submerged, glacier, cave, on, peri...    0.0  \n",
       "1  [the, ice, facade, is, approximately, 60, m, h...    0.0  \n",
       "2  [ice, formations, in, the, titlis, glacier, cave]    0.0  \n",
       "3  [a, glacier, cave, is, a, cave, formed, within...    1.0  \n",
       "4  [glacier, caves, are, often, called, ice, cave...    0.0  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack = mz.datasets.toy.load_data()\n",
    "chain = mz.chain_transform([mz.processor_units.TokenizeUnit(),\n",
    "                           mz.processor_units.LowercaseUnit()])\n",
    "data_pack.apply_on_text(chain, inplace=True)\n",
    "data_pack.frame[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that some processor units are stateful so we have to `fit` them before using their `transform`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.494898Z",
     "start_time": "2019-01-11T16:14:13.490454Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matchzoo.processor_units.processor_units.StatefulProcessorUnit"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mz.processor_units.VocabularyUnit.__base__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.511427Z",
     "start_time": "2019-01-11T16:14:13.497050Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_unit = mz.processor_units.VocabularyUnit()\n",
    "texts = data_pack.frame()[['text_left', 'text_right']]\n",
    "all_tokens = texts.sum().sum()\n",
    "vocab_unit.fit(all_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Such `StatefulProcessorUnit` will save information in its `state` when `fit`, similar to the `context` of a preprocessor. In our case here, the vocabulary unit will save a term to index mapping, and a index to term mapping, called `term_index` and `index_term` respectively. Then we can proceed transforming a data pack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.517844Z",
     "start_time": "2019-01-11T16:14:13.513565Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "how 795\n",
      "are 726\n",
      "glacier 874\n"
     ]
    }
   ],
   "source": [
    "for vocab in 'how', 'are', 'glacier':\n",
    "    print(vocab, vocab_unit.state['term_index'][vocab])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.547527Z",
     "start_time": "2019-01-11T16:14:13.520308Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 13611.07it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 49707.32it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[795, 726, 874, 313, 771, 808]</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[440, 784, 832, 874, 47, 890, 160, 207, 874, 223]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[795, 726, 874, 313, 771, 808]</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[91, 228, 167, 26, 242, 351, 343, 866]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[795, 726, 874, 313, 771, 808]</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[228, 442, 201, 91, 230, 874, 47]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[795, 726, 874, 313, 771, 808]</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[440, 874, 47, 26, 440, 47, 771, 892, 91, 228,...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[795, 726, 874, 313, 771, 808]</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[874, 313, 726, 36, 293, 228, 313, 703, 543, 8...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                       text_left id_right  \\\n",
       "0      Q1  [795, 726, 874, 313, 771, 808]     D1-0   \n",
       "1      Q1  [795, 726, 874, 313, 771, 808]     D1-1   \n",
       "2      Q1  [795, 726, 874, 313, 771, 808]     D1-2   \n",
       "3      Q1  [795, 726, 874, 313, 771, 808]     D1-3   \n",
       "4      Q1  [795, 726, 874, 313, 771, 808]     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [440, 784, 832, 874, 47, 890, 160, 207, 874, 223]    0.0  \n",
       "1             [91, 228, 167, 26, 242, 351, 343, 866]    0.0  \n",
       "2                  [228, 442, 201, 91, 230, 874, 47]    0.0  \n",
       "3  [440, 874, 47, 26, 440, 47, 771, 892, 91, 228,...    1.0  \n",
       "4  [874, 313, 726, 36, 293, 228, 313, 703, 543, 8...    0.0  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack.apply_on_text(vocab_unit.transform, inplace=True)\n",
    "data_pack.frame()[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since this usage is quite common, we wrapped a function to do the same thing. For other stateful units, consult their documentations and try `mz.build_unit_from_data_pack`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:13.606898Z",
     "start_time": "2019-01-11T16:14:13.549257Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with extend: 100%|██████████| 13/13 [00:00<00:00, 22766.58it/s]\n",
      "Processing text_right with extend: 100%|██████████| 100/100 [00:00<00:00, 116153.53it/s]\n",
      "Building VocabularyUnit from a datapack.: 100%|██████████| 13893/13893 [00:00<00:00, 2849181.77it/s]\n",
      "Processing text_left with transform: 100%|██████████| 13/13 [00:00<00:00, 6848.27it/s]\n",
      "Processing text_right with transform: 100%|██████████| 100/100 [00:00<00:00, 19253.17it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>text_left</th>\n",
       "      <th>id_right</th>\n",
       "      <th>text_right</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...</td>\n",
       "      <td>D1-0</td>\n",
       "      <td>[69, 71, 40, 17, 19, 41, 67, 44, 71, 23, 33, 2...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...</td>\n",
       "      <td>D1-1</td>\n",
       "      <td>[12, 22, 58, 71, 62, 53, 58, 71, 65, 17, 53, 1...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...</td>\n",
       "      <td>D1-2</td>\n",
       "      <td>[32, 53, 58, 71, 65, 63, 19, 35, 17, 41, 62, 6...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...</td>\n",
       "      <td>D1-3</td>\n",
       "      <td>[69, 71, 55, 67, 17, 53, 62, 58, 19, 71, 53, 1...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q1</td>\n",
       "      <td>[22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...</td>\n",
       "      <td>D1-4</td>\n",
       "      <td>[51, 67, 17, 53, 62, 58, 19, 71, 53, 17, 4, 58...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_left                                          text_left id_right  \\\n",
       "0      Q1  [22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...     D1-0   \n",
       "1      Q1  [22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...     D1-1   \n",
       "2      Q1  [22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...     D1-2   \n",
       "3      Q1  [22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...     D1-3   \n",
       "4      Q1  [22, 63, 45, 71, 17, 19, 58, 71, 55, 67, 17, 5...     D1-4   \n",
       "\n",
       "                                          text_right  label  \n",
       "0  [69, 71, 40, 17, 19, 41, 67, 44, 71, 23, 33, 2...    0.0  \n",
       "1  [12, 22, 58, 71, 62, 53, 58, 71, 65, 17, 53, 1...    0.0  \n",
       "2  [32, 53, 58, 71, 65, 63, 19, 35, 17, 41, 62, 6...    0.0  \n",
       "3  [69, 71, 55, 67, 17, 53, 62, 58, 19, 71, 53, 1...    1.0  \n",
       "4  [51, 67, 17, 53, 62, 58, 19, 71, 53, 17, 4, 58...    0.0  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pack = mz.datasets.toy.load_data()\n",
    "vocab_unit = mz.build_vocab_unit(data_pack)\n",
    "data_pack.apply_on_text(vocab_unit.transform).frame[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T14:34:11.732867Z",
     "start_time": "2019-01-11T14:34:11.730142Z"
    }
   },
   "source": [
    "## Data Generators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes your memory just can't hold all transformed data. One choice is to transform and fit in smaller chunks of data. MatchZoo provides `DataGenerator` as an alternative. Instead of `fit`, you may do a `fit_generator` that takes a data generator that `unpack` data on the fly. `DynamicDataGenerator` is especially useful when you want to delay a part of the preprocessing process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-11T16:14:16.357083Z",
     "start_time": "2019-01-11T16:14:13.608612Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "100/100 [==============================] - 2s 17ms/step - loss: 0.0491\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x135430ef0>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = mz.datasets.toy.load_data()\n",
    "preprocessor = mz.preprocessors.DSSMPreprocessor(with_word_hashing=False)\n",
    "data = preprocessor.fit_transform(data, verbose=0)\n",
    "\n",
    "model = mz.models.DSSM()\n",
    "model.params['input_shapes'] = preprocessor.context['input_shapes']\n",
    "model.guess_and_fill_missing_params(verbose=0)\n",
    "model.build()\n",
    "model.compile()\n",
    "\n",
    "term_index = preprocessor.context['vocab_unit'].state['term_index']\n",
    "hashing_unit = mz.processor_units.WordHashingUnit(term_index)\n",
    "data_generator = mz.DynamicDataGenerator(hashing_unit.transform, data, batch_size=1)\n",
    "\n",
    "model.fit_generator(data_generator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, losses like `RankHingeLoss` and `RankCrossEntropyLoss` have to be used with `PairDataGenerator`, since batch-wise information are needed and computed on the fly."
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "matchzoo",
   "language": "python",
   "name": "matchzoo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "notify_time": "5",
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "171px",
    "left": "2190px",
    "right": "20px",
    "top": "120px",
    "width": "349px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
